{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TN(y_true, y_predict):\n",
    "    assert len(y_true) == len(y_predict)\n",
    "    return np.sum((y_true == 0) & (y_predict==0))   # 注意这里是一个‘&’\n",
    "\n",
    "def FP(y_true, y_predict):\n",
    "    assert len(y_true) == len(y_predict)\n",
    "    return np.sum((y_true == 0) & (y_predict==1))\n",
    "\n",
    "def FN(y_true, y_predict):\n",
    "    assert len(y_true) == len(y_predict)\n",
    "    return np.sum((y_true == 1) & (y_predict==0))\n",
    "\n",
    "def TP(y_true, y_predict):\n",
    "    assert len(y_true) == len(y_predict)\n",
    "    return np.sum((y_true == 1) & (y_predict==1))\n",
    "\n",
    "def confusion_matrix(y_true, y_predict):\n",
    "    return np.array([\n",
    "        [TN(y_true, y_predict), FP(y_true, y_predict)],\n",
    "        [FN(y_true, y_predict), TP(y_true, y_predict)]\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precision_score(y_true, y_predict):\n",
    "    tp = TP(y_true, y_predict)\n",
    "    fp = FP(y_true, y_predict)\n",
    "    try:\n",
    "        return tp / (tp + fp)\n",
    "    except:   # 处理分母为0的情况\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall_score(y_true, y_predict):\n",
    "    tp = TP(y_true, y_predict)\n",
    "    fn = FN(y_true, y_predict)\n",
    "\n",
    "    try:\n",
    "        return tp / (tp + fn)\n",
    "    except:\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TPR(y_true, y_predict):\n",
    "    tp = TP(y_true, y_predict)\n",
    "    fn = FN(y_true, y_predict)\n",
    "\n",
    "    try:\n",
    "        return tp / (tp + fn)\n",
    "    except:\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def FPR(y_true, y_predict):\n",
    "    fp = FP(y_true, y_predict)\n",
    "    tn = TN(y_true, y_predict)\n",
    "\n",
    "    try:\n",
    "        return fp / (fp + tn)\n",
    "    except:\n",
    "        return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yinahe\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target.copy()\n",
    "\n",
    "y[digits.target==9] = 1\n",
    "y[digits.target!=9] = 0\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "decision_scores = log_reg.decision_function(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAQrUlEQVR4nO3dfZBddX3H8fc3GxKQh4Ds8tA8uLEGywq0oTsMrW1FozSEmWTsWCexjNWhZrTF/qHtDEpLHZz+Ua064xAfMtahOiOIzlR2JMpYDaW1RlgHDSQ0uEYwayLZUAxgCHnYb/+4F3uzuck9Se7u3fvb92tmZ+4555d7P7/czYfDueecG5mJJKn7zep0AElSe1joklQIC12SCmGhS1IhLHRJKsTsTr1wb29v9vf3d+rlJakr/eAHP9iTmX3NtnWs0Pv7+xkeHu7Uy0tSV4qIJ4+1zUMuklQIC12SCmGhS1IhLHRJKoSFLkmFaFnoEfH5iNgdEY8eY3tExCcjYiQiNkfEle2PKUlqpcoe+h3A8uNsvw5YUv9ZC3z61GNJkk5Uy/PQM/OBiOg/zpBVwBeydh/eTRFxbkRcnJm72pRRJyEzufPBHfxi7wudjiJpgmWXXshvLzy37c/bjguL5gM7GpZH6+uOKvSIWEttL55Fixa14aV1LFt3PcsH/+0RACI6HEbSES445/RpW+jN6qLpt2Zk5npgPcDg4KDfrDGJ7t28i55ZwYMfXMb5Z83tdBxJU6AdZ7mMAgsblhcAO9vwvDpJmcm9j+zi93/zfMtcmkHaUehDwNvrZ7tcDez1+PnkGx9PDh0eb/rzyM/38uTT+1hx+cWdjilpCrU85BIRdwLXAL0RMQr8A3AaQGZ+BtgArABGgH3AOycrrGr2vnCQZR/7D/Y8/+Ixx/TMCv74NRdNYSpJnVblLJc1LbYn8FdtS6SWvrX1KfY8/yJ/8QeLmXfGaU3HLLnwLF5+5pwpTiapkzp2+1ydvHs372TBeWdwy/WXEp7CIqnOS/+7zN59B/mvkT1cf/nFlrmkI7iH3uCbj+7ikZ/v7XSM43ri6X0cPJxcf4UfeEo6koXe4O/v2cKe51+kZ5rv+S5ddC6Xz5/X6RiSphkLvUFm8rarFvGPb76801Ek6YR5DF2SCjFj99APHR7ncB5594H0ZgSSutiMLPSnnt3PG/75fn514PBR22bPmt7HzyXpWGZkoT/8s1/yqwOHeedr++ltuNdJBFzv5fKSutSMLPQfP/UcAH9z7as5c+6M/CuQVKAZ+aHo47ufZ/65Z1jmkooyoxpt97P7+cL3nuTBnz7NwMXndDqOJLXVjCr0b275BbdvHGHu7Fm8/rcu6HQcSWqrGVXo4+O18xI3fWAZ53knQkmFmZHH0CWpRMXvoR84NM6h8fHa48PjHU4jSZOn6ELf/ex+XvfR+3nh4JEXEM3y4iFJBSq60Pc8f4AXDh7mT66cz6svPBuAi+adfsxv+ZGkblZ0ob/k2oGLWH6Z368pqWx+KCpJhShiD/3g4XE+ff9PePaFg0esf/pXBzqUSJKmXhGF/vhTz/Hxbz3O3Nmzjrpb4vlnzqG/92UdSiZJU6eIQn/pPua3v+1K3jRwYWfDSFKHeAxdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFaJrT1s8cGic27/zY57df8gLiCSJLi70bb94jk9+Z4QzTuvhtJ7gwnPm0n++FxBJmrm6ttCT2tVEt79tKcsu9WIiSap0DD0ilkfEtogYiYibm2xfFBEbI+LhiNgcESvaH1WSdDwtCz0ieoB1wHXAALAmIgYmDPs74O7MXAqsBj7V7qATHfTbhyTpCFX20K8CRjJze2YeAO4CVk0Yk8A59cfzgJ3ti9jcDZ97EIDZPZ6oI0lQrdDnAzsalkfr6xp9CLghIkaBDcB7mz1RRKyNiOGIGB4bGzuJuP/v0Pg4vWfN5epXvvyUnkeSSlGl0Jt9AWdOWF4D3JGZC4AVwBcj4qjnzsz1mTmYmYN9fX0nnrbBaT2zePPS32Du7J5Teh5JKkWVQh8FFjYsL+DoQyo3AncDZOb3gNOB3nYElCRVU6XQHwKWRMTiiJhD7UPPoQljfgYsA4iIS6kV+qkdU5EknZCWhZ6Zh4CbgPuAx6idzbIlIm6LiJX1Ye8H3hURPwLuBN6RmRMPy0iSJlGlC4sycwO1Dzsb193a8Hgr8Nr2RpMknQjP+ZOkQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQXVnoDzw+xr4DhzsdQ5Kmla4s9Dv++wkAfmfheZ0NIknTSFcWemZyxYJ5XH/FxZ2OIknTRlcWuiTpaBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgpRqdAjYnlEbIuIkYi4+Rhj3hoRWyNiS0R8qb0xJUmtzG41ICJ6gHXAm4BR4KGIGMrMrQ1jlgAfAF6bmc9ExAWTFViS1FyVPfSrgJHM3J6ZB4C7gFUTxrwLWJeZzwBk5u72xpQktVKl0OcDOxqWR+vrGl0CXBIR342ITRGxvNkTRcTaiBiOiOGxsbGTSyxJaqpKoUeTdTlheTawBLgGWAN8LiLOPeoPZa7PzMHMHOzr6zvRrJKk46hS6KPAwoblBcDOJmPuycyDmflTYBu1gpckTZEqhf4QsCQiFkfEHGA1MDRhzNeA1wNERC+1QzDb2xlUknR8LQs9Mw8BNwH3AY8Bd2fmloi4LSJW1ofdBzwdEVuBjcDfZubTkxVaknS0lqctAmTmBmDDhHW3NjxO4H31H0lSB3ilqCQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCtF1hf74U8+xcdsYmZ1OIknTS9cV+sb/2Q3AHy7p7XASSZpeuq7QX3LTG17V6QiSNK10baFLko5koUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVIhKhR4RyyNiW0SMRMTNxxn3lojIiBhsX0RJUhUtCz0ieoB1wHXAALAmIgaajDsb+Gvg++0OKUlqrcoe+lXASGZuz8wDwF3AqibjPgx8BNjfxnySpIqqFPp8YEfD8mh93a9FxFJgYWZ+vY3ZJEknoEqhR5N1v755bUTMAj4BvL/lE0WsjYjhiBgeGxurnlKS1FKVQh8FFjYsLwB2NiyfDVwG3B8RTwBXA0PNPhjNzPWZOZiZg319fSefWpJ0lCqF/hCwJCIWR8QcYDUw9NLGzNybmb2Z2Z+Z/cAmYGVmDk9KYklSUy0LPTMPATcB9wGPAXdn5paIuC0iVk52QElSNbOrDMrMDcCGCetuPcbYa049liTpRHmlqCQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQlQo9IpZHxLaIGImIm5tsf19EbI2IzRHx7Yh4RfujSpKOp2WhR0QPsA64DhgA1kTEwIRhDwODmXkF8FXgI+0OKkk6vip76FcBI5m5PTMPAHcBqxoHZObGzNxXX9wELGhvTElSK1UKfT6wo2F5tL7uWG4EvtFsQ0SsjYjhiBgeGxurnlKS1FKVQo8m67LpwIgbgEHgo822Z+b6zBzMzMG+vr7qKSVJLc2uMGYUWNiwvADYOXFQRLwRuAV4XWa+2J54kqSqquyhPwQsiYjFETEHWA0MNQ6IiKXAZ4GVmbm7/TElSa20LPTMPATcBNwHPAbcnZlbIuK2iFhZH/ZR4CzgKxHxw4gYOsbTSZImSZVDLmTmBmDDhHW3Njx+Y5tzSZJOkFeKSlIhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJ0XaEv7j2TFZdfxKyITkeRpGmlUqFHxPKI2BYRIxFxc5PtcyPiy/Xt34+I/nYHfcm1r7mIT/3Z73L6aT2T9RKS1JVaFnpE9ADrgOuAAWBNRAxMGHYj8Exmvgr4BPBP7Q4qSTq+KnvoVwEjmbk9Mw8AdwGrJoxZBfxr/fFXgWURHhORpKlUpdDnAzsalkfr65qOycxDwF7g/IlPFBFrI2I4IobHxsZOLrEkqakqhd5sTztPYgyZuT4zBzNzsK+vr0o+SVJFVQp9FFjYsLwA2HmsMRExG5gH/G87AkqSqqlS6A8BSyJicUTMAVYDQxPGDAF/Xn/8FuA7mXnUHrokafLMbjUgMw9FxE3AfUAP8PnM3BIRtwHDmTkE/AvwxYgYobZnvnoyQ0uSjtay0AEycwOwYcK6Wxse7wf+tL3RJEknIjp1ZCQixoAnT/KP9wJ72hinGzjnmcE5zwynMudXZGbTs0o6VuinIiKGM3Ow0zmmknOeGZzzzDBZc+66e7lIkpqz0CWpEN1a6Os7HaADnPPM4JxnhkmZc1ceQ5ckHa1b99AlSRNY6JJUiGld6NPpizWmSoU5vy8itkbE5oj4dkS8ohM526nVnBvGvSUiMiK6/hS3KnOOiLfW3+stEfGlqc7YbhV+txdFxMaIeLj++72iEznbJSI+HxG7I+LRY2yPiPhk/e9jc0RcecovmpnT8ofabQZ+ArwSmAP8CBiYMOYvgc/UH68Gvtzp3FMw59cDL6s/fs9MmHN93NnAA8AmYLDTuafgfV4CPAycV1++oNO5p2DO64H31B8PAE90OvcpzvmPgCuBR4+xfQXwDWp3q70a+P6pvuZ03kOfiV+s0XLOmbkxM/fVFzdRu/tlN6vyPgN8GPgIsH8qw02SKnN+F7AuM58ByMzdU5yx3arMOYFz6o/ncfRdXbtKZj7A8e86uwr4QtZsAs6NiItP5TWnc6G37Ys1ukiVOTe6kdp/4btZyzlHxFJgYWZ+fSqDTaIq7/MlwCUR8d2I2BQRy6cs3eSoMucPATdExCi1e0e9d2qidcyJ/ntvqdLNuTqkbV+s0UUqzycibgAGgddNaqLJd9w5R8Qsat9T+46pCjQFqrzPs6kddrmG2v+F/WdEXJaZv5zkbJOlypzXAHdk5sci4veo3cH1sswcn/x4HdH2/prOe+gz8Ys1qsyZiHgjcAuwMjNfnKJsk6XVnM8GLgPuj4gnqB1rHOryD0ar/m7fk5kHM/OnwDZqBd+tqsz5RuBugMz8HnA6tZtYlarSv/cTMZ0LfSZ+sUbLOdcPP3yWWpl3+3FVaDHnzNybmb2Z2Z+Z/dQ+N1iZmcOdidsWVX63v0btA3AiopfaIZjtU5qyvarM+WfAMoCIuJRaoZf85cNDwNvrZ7tcDezNzF2n9Iyd/iS4xafEK4DHqX06fkt93W3U/kFD7Q3/CjACPAi8stOZp2DO/w48Bfyw/jPU6cyTPecJY++ny89yqfg+B/BxYCvwCLC605mnYM4DwHepnQHzQ+DaTmc+xfneCewCDlLbG78ReDfw7ob3eF397+ORdvxee+m/JBViOh9ykSSdAAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFeL/AEbAv/28z/m4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fprs = []\n",
    "tprs = []\n",
    "\n",
    "thresholds = np.arange(np.min(decision_scores), np.max(decision_scores), step=0.1)\n",
    "for threshold in thresholds:\n",
    "    y_predict = np.array(decision_scores >= threshold, dtype='int')\n",
    "    fprs.append(FPR(y_test, y_predict))\n",
    "    tprs.append(TPR(y_test, y_predict))\n",
    "\n",
    "plt.plot(fprs, tprs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### scikit中的ROC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fprs, tprs, thresholds = roc_curve(y_test, decision_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAO80lEQVR4nO3db4xcV33G8e9D3BRVTaDURgr+g43qSCxRRdAqDkIqQdDKiYT9hiIbRZQqwoU29AWoUiqqgMKrglokJLdgtYiCFELgBbGQUaTSRFQIu1kUE7AjV1sHkk2iZiFpeIEgRP31xYzJdD27c9ee3dk5+/1IK8299+zM73hmn5ycc+/cVBWSpOn3skkXIEkaDwNdkhphoEtSIwx0SWqEgS5JjdgyqRfeunVr7d69e1IvL0lT6Xvf+95PqmrbsGMTC/Tdu3czNzc3qZeXpKmU5MfLHXPKRZIaYaBLUiMMdElqhIEuSY0w0CWpESMDPcnnkzyT5IfLHE+SzySZT/JIkjeNv0xJ0ihdRuhfAPavcPxmYG//5wjwj5dfliRptUaeh15V306ye4UmB4EvVu97eE8meWWSa6rq6THVqEt096nHue/0k5MuQ9ISM6+5mo+98w1jf95xzKFvB54Y2F7o77tIkiNJ5pLMLS4ujuGltZL7Tj/J2ad/NukyJK2TcVwpmiH7ht41o6qOAccAZmdnvbPGOpi55mq+8mdvnnQZktbBOEboC8DOge0dwFNjeF5J0iqMI9CPA+/tn+1yI/C88+eStP5GTrkk+TJwE7A1yQLwMeA3AKrqs8AJ4BZgHvg58KdrVaxWt9B59umfMXPN1WtckaSNostZLodHHC/gL8ZWkVZ0YaGzS1DPXHM1B984dH1aUoMm9vW5unQudEoaxkv/JakRjtAHTMOFOM6LS1qOI/QB03AhjvPikpbjCH0J56clTStH6JLUiE07Qh82X+78tKRptmlH6MPmy52fljTNNu0IHZwvl9SWTTtCl6TWGOiS1IhNNeUyuBDqAqik1myqEfrgQqgLoJJas6lG6OBCqKR2baoRuiS1rPkRuvPmkjaL5kfozptL2iyaH6GD8+aSNofmR+iStFkY6JLUiGamXJa725ALoZI2i2ZG6MvdbciFUEmbRTMjdHDxU9Lm1swIXZI2OwNdkhphoEtSIwx0SWqEgS5JjZjqs1z84i1JeslUj9D94i1JeslUj9DBc88l6YJOI/Qk+5OcSzKf5I4hx3cleSDJw0keSXLL+EuVJK1kZKAnuQI4CtwMzACHk8wsafY3wL1VdT1wCPiHcRcqSVpZlxH6DcB8VZ2vqheAe4CDS9oUcGFF8hXAU+Mrcbi7Tz3OqceeXeuXkaSp0SXQtwNPDGwv9PcN+jhwa5IF4ATwoWFPlORIkrkkc4uLi5dQ7ksunN3iQqgk9XQJ9AzZV0u2DwNfqKodwC3Al5Jc9NxVdayqZqtqdtu2bauvdol9e17Fe/btuuznkaQWdAn0BWDnwPYOLp5SuQ24F6Cqvgu8HNg6jgIlSd10CfSHgL1J9iS5kt6i5/ElbR4H3g6Q5PX0Av3y5lQkSasyMtCr6kXgduB+4FF6Z7OcSXJXkgP9Zh8B3p/k+8CXgfdV1dJpGUnSGup0YVFVnaC32Dm4786Bx2eBt4y3NEnSakz1pf+SpJcY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEVMZ6HefepxTjz076TIkaUOZykC/7/STABx84/YJVyJJG8dUBjrAvj2v4j37dk26DEnaMKY20CVJ/5+BLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNaJToCfZn+RckvkkdyzT5t1JziY5k+Tu8ZYpSRply6gGSa4AjgJ/CCwADyU5XlVnB9rsBf4aeEtVPZfk1WtVsCRpuC4j9BuA+ao6X1UvAPcAB5e0eT9wtKqeA6iqZ8ZbpiRplC6Bvh14YmB7ob9v0LXAtUm+k+Rkkv3DnijJkSRzSeYWFxcvrWJJ0lBdAj1D9tWS7S3AXuAm4DDwT0leedEvVR2rqtmqmt22bdtqa5UkraBLoC8AOwe2dwBPDWlzX1X9qqoeA87RC3hJ0jrpEugPAXuT7ElyJXAIOL6kzdeBtwEk2UpvCub8OAuVJK1sZKBX1YvA7cD9wKPAvVV1JsldSQ70m90P/DTJWeAB4K+q6qdrVbQk6WIjT1sEqKoTwIkl++4ceFzAh/s/kqQJ8EpRSWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1YuoC/e5Tj3PqsWcnXYYkbThTF+j3nX4SgINv3D7hSiRpY5m6QAfYt+dVvGffrkmXIUkbylQGuiTpYga6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmN6BToSfYnOZdkPskdK7R7V5JKMju+EiVJXYwM9CRXAEeBm4EZ4HCSmSHtrgL+Ejg17iIlSaN1GaHfAMxX1fmqegG4Bzg4pN0ngE8CvxhjfZKkjroE+nbgiYHthf6+X0tyPbCzqr6x0hMlOZJkLsnc4uLiqouVJC2vS6BnyL769cHkZcCngY+MeqKqOlZVs1U1u23btu5VSpJG6hLoC8DOge0dwFMD21cB1wEPJvkRcCNw3IVRSVpfXQL9IWBvkj1JrgQOAccvHKyq56tqa1XtrqrdwEngQFXNrUnFkqShRgZ6Vb0I3A7cDzwK3FtVZ5LcleTAWhcoSepmS5dGVXUCOLFk353LtL3p8suSJK2WV4pKUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRnQK9CT7k5xLMp/kjiHHP5zkbJJHknwryWvHX6okaSUjAz3JFcBR4GZgBjicZGZJs4eB2ar6feBrwCfHXagkaWVdRug3APNVdb6qXgDuAQ4ONqiqB6rq5/3Nk8CO8ZYpSRqlS6BvB54Y2F7o71vObcA3hx1IciTJXJK5xcXF7lVKkkbqEugZsq+GNkxuBWaBTw07XlXHqmq2qma3bdvWvUpJ0khbOrRZAHYObO8AnlraKMk7gI8Cb62qX46nPElSV11G6A8Be5PsSXIlcAg4PtggyfXA54ADVfXM+MuUJI0yMtCr6kXgduB+4FHg3qo6k+SuJAf6zT4F/Dbw1SSnkxxf5ukkSWuky5QLVXUCOLFk350Dj98x5rokSavklaKS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGmGgS1IjDHRJaoSBLkmNMNAlqREGuiQ1wkCXpEYY6JLUCANdkhphoEtSIwx0SWqEgS5JjTDQJakRBrokNcJAl6RGGOiS1AgDXZIaYaBLUiMMdElqhIEuSY0w0CWpEQa6JDWiU6An2Z/kXJL5JHcMOf6bSb7SP34qye5xFypJWtnIQE9yBXAUuBmYAQ4nmVnS7Dbguar6PeDTwN+Ou1BJ0sq6jNBvAOar6nxVvQDcAxxc0uYg8C/9x18D3p4k4ytTkjTKlg5ttgNPDGwvAPuWa1NVLyZ5Hvhd4CeDjZIcAY4A7Nq165IKnnnN1Zf0e5LUui6BPmykXZfQhqo6BhwDmJ2dveh4Fx975xsu5dckqXldplwWgJ0D2zuAp5Zrk2QL8Arg2XEUKEnqpkugPwTsTbInyZXAIeD4kjbHgT/pP34X8G9VdUkjcEnSpRk55dKfE78duB+4Avh8VZ1JchcwV1XHgX8GvpRknt7I/NBaFi1JuliXOXSq6gRwYsm+Owce/wL44/GWJklaDa8UlaRGGOiS1AgDXZIaYaBLUiMyqbMLkywCP77EX9/KkqtQNwH7vDnY583hcvr82qraNuzAxAL9ciSZq6rZSdexnuzz5mCfN4e16rNTLpLUCANdkhoxrYF+bNIFTIB93hzs8+awJn2eyjl0SdLFpnWELklawkCXpEZs6EDfjDen7tDnDyc5m+SRJN9K8tpJ1DlOo/o80O5dSSrJ1J/i1qXPSd7df6/PJLl7vWsctw6f7V1JHkjycP/zfcsk6hyXJJ9P8kySHy5zPEk+0//3eCTJmy77RatqQ/7Q+6re/wJeB1wJfB+YWdLmz4HP9h8fAr4y6brXoc9vA36r//iDm6HP/XZXAd8GTgKzk657Hd7nvcDDwO/0t1896brXoc/HgA/2H88AP5p03ZfZ5z8A3gT8cJnjtwDfpHfHtxuBU5f7mht5hL4Zb049ss9V9UBV/by/eZLeHaSmWZf3GeATwCeBX6xncWukS5/fDxytqucAquqZda5x3Lr0uYALNw1+BRffGW2qVNW3WfnObQeBL1bPSeCVSa65nNfcyIE+7ObU25drU1UvAhduTj2tuvR50G30/gs/zUb2Ocn1wM6q+sZ6FraGurzP1wLXJvlOkpNJ9q9bdWujS58/DtyaZIHe/Rc+tD6lTcxq/95H6nSDiwkZ282pp0jn/iS5FZgF3rqmFa29Ffuc5GXAp4H3rVdB66DL+7yF3rTLTfT+L+zfk1xXVf+zxrWtlS59Pgx8oar+Lsmb6d0F7bqq+t+1L28ixp5fG3mEvhlvTt2lzyR5B/BR4EBV/XKdalsro/p8FXAd8GCSH9Gbazw+5QujXT/b91XVr6rqMeAcvYCfVl36fBtwL0BVfRd4Ob0vsWpVp7/31djIgb4Zb049ss/96YfP0QvzaZ9XhRF9rqrnq2prVe2uqt301g0OVNXcZModiy6f7a/TWwAnyVZ6UzDn17XK8erS58eBtwMkeT29QF9c1yrX13Hgvf2zXW4Enq+qpy/rGSe9EjxilfgW4D/prY5/tL/vLnp/0NB7w78KzAP/Abxu0jWvQ5//Ffhv4HT/5/ika17rPi9p+yBTfpZLx/c5wN8DZ4EfAIcmXfM69HkG+A69M2BOA3806Zovs79fBp4GfkVvNH4b8AHgAwPv8dH+v8cPxvG59tJ/SWrERp5ykSStgoEuSY0w0CWpEQa6JDXCQJekRhjoktQIA12SGvF/IC5d6mFx4nMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fprs, tprs)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
